/*
 * TOP SECRET Copyright 2006-2015 Transsion.com All right reserved. This software is the confidential and proprietary
 * information of Transsion.com ("Confidential Information"). You shall not disclose such Confidential Information and
 * shall use it only in accordance with the terms of the license agreement you entered into with Transsion.com.
 */
package com.yunji.framework_template.books.biz.service;

import java.io.IOException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import javax.annotation.Resource;

import org.apache.commons.httpclient.HttpException;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.stereotype.Service;

import com.yunji.framework_template.books.biz.vo.BooksVO;
import com.yunji.framework_template.books.biz.vo.ChaptersVO;
import com.yunji.framework_template.books.orm.persistence.model.Books;
import com.yunji.framework_template.books.orm.persistence.model.BooksChapters;
import com.yunji.framework_template.books.orm.persistence.model.BooksCrawled;
import com.yunji.framework_template.common.http.JsoupUtil;
import com.yunji.framework_template.common.util.MD5Util;
import com.yunji.framework_template.common.util.StringUtil;
import lombok.extern.slf4j.Slf4j;

/**
 * ClassName:CrawleBooks <br/>
 * Date: 2019年1月16日 下午5:43:01 <br/>
 * 
 * @author fenglibin1982@163.com
 * @Blog http://blog.csdn.net/fenglibing
 * @version
 * @see
 */
@Slf4j
@Service
public class BooksCrawlerService {

    private String               baseUrl   = "http://books.framework_template.cc/meta/titles/%c.html";
    // 保存已经抓取过的URL
    private Set<String>          booksSet  = new HashSet<String>();
    private static boolean       isRunning = false;

    @Resource
    private BooksService         bookService;
    @Resource
    private BooksChaptersService bookChapterService;
    @Resource
    private BooksCrawledService  booksCrawledService;

    public void crawleBooks() throws HttpException, IOException {
        if (isRunning) {
            log.info("Book crawler is running.");
            return;
        }

        try {
            isRunning = true;
            for (int start = 65; start <= 90; start++) {// 从字母A-Z
                String bookUrl = String.format(baseUrl, (char) start);
                Element contentElement = JsoupUtil.getPageElement(bookUrl);
                if (contentElement == null) {
                    continue;
                }
                Elements bookElements = contentElement.select("ul[class=\"works\"]>li>a");
                handleBooksElement(bookElements);
            }
        } finally {
            isRunning = false;
        }
    }

    private void handleBooksElement(Elements bookElements) {
        bookElements.forEach(e -> {
            try {
                handleOneBookElement(e, true);
            } catch (Throwable t) {
                log.error("Handle book exception happened:" + e.absUrl("href"), t);
            }
        });
    }

    private boolean isChapterUrl(String url) {
        String regEx = ".*chapter[1-9]{1,}\\.html.*";
        Pattern pattern = Pattern.compile(regEx);
        Matcher matcher = pattern.matcher(url);
        return matcher.matches();
    }

    private void handleOneBookElement(Element bookElement, boolean recursive) {
        String url = bookElement.absUrl("href");
        /*
         * if(isChapterUrl(url)) {//书的列表中如果包含了其它书的某些章节的链接，忽略掉不处理 return; }
         */
        String urlMd5 = MD5Util.md5Hex(url);
        if (booksSet.contains(urlMd5)) {
            log.info("Current book has already crawled:" + url);
            return;
        }
        BooksCrawled booksCrawled = booksCrawledService.selectByUrlMd5(urlMd5);
        if (booksCrawled != null) {
            log.info("Current book has already crawled:" + url);
            return;
        }
        BooksVO book = handleOneBook(bookElement, recursive);
        if (book != null && book.getChapters() != null && book.getChapters().size() >= 1) {
            insert(book);
            booksSet.add(urlMd5);
            booksCrawled = new BooksCrawled();
            booksCrawled.setUrl(url);
            booksCrawled.setUrlMd5(urlMd5);
            booksCrawledService.insert(booksCrawled);
            log.warn("Book crawled successfully: " + url);
        } else {
            log.warn("Book crawled fail: " + url);
        }
    }

    public BooksVO handleOneBook(Element bookElement, boolean recursive) {

        String title = bookElement.ownText();
        String url = bookElement.absUrl("href");
        if (!StringUtil.isEmpty(url)) {
            String bookTitle = "";
            String bookAuthor = "";
            String[] titleArr = title.split("/");
            String publishedYear = StringUtil.subString(title, "[", "]");
            if (titleArr.length >= 2) {
                bookTitle = titleArr[0];
                bookAuthor = titleArr[1];
                int yIndex = bookAuthor.indexOf("[");
                if (yIndex > 0) {// 包括了年份，将年份去掉
                    bookAuthor = bookAuthor.substring(0, yIndex);
                }
            } else {
                bookTitle = title;
                int yIndex = bookTitle.indexOf("[");
                if (yIndex > 0) {// 包括了年份，将年份去掉
                    bookTitle = bookTitle.substring(0, yIndex);
                }
            }
            BooksVO book = BooksVO.builder().name(bookTitle).author(bookAuthor).published(publishedYear).build();
            try {
                Element indexElement = JsoupUtil.getPageElement(url);
                if (indexElement == null) {
                    return null;
                }
                String indexHtml = indexElement.html();

                String keywords = StringUtil.subString(indexHtml, "\"keywords\" : \"", "\",");
                String language = StringUtil.subString(indexHtml, "\"inLanguage\" : \"", "\",");
                String datePublished = StringUtil.subString(indexHtml, "\"datePublished\" : \"", "\",");
                String author2 = StringUtil.subString(indexHtml, "\"author\" : \"", "\",");
                String description = StringUtil.subString(indexHtml, "\"description\" : \"", "\",");
                String cover = StringUtil.subString(indexHtml, "\"image\" : \"", "\",");
                book.setKeywords(keywords);
                book.setLanguage(language);
                if (StringUtil.isEmpty(book.getPublished())) {
                    book.setPublished(datePublished);
                }
                if (StringUtil.isEmpty(book.getAuthor())) {
                    book.setAuthor(author2);
                }
                if (cover != null && !cover.endsWith("cover.jpg")) {
                    book.setCover(cover);
                }
                book.setIntroduce(description);
                final List<ChaptersVO> chaptersList = new ArrayList<ChaptersVO>();

                Elements indexContentElements = null;
                Elements indexChapterElements = indexElement.select("div[class=\"chapter\"]");
                Elements indexActElements = indexElement.select("div[class=\"act\"]");
                Elements indexFrontmatterElements = indexElement.select("div[class=\"frontmatter\"]");
                Elements indexEssayElements = indexElement.select("div[class=\"essay\"]");
                Elements contentsElements = indexElement.select("div[class=\"contents\"]");
                Elements bodyElements = indexElement.select("div[class=\"body\"]");
                Elements indexWorksElements = indexElement.select("div[class=\"works\"]");
                Elements indexStoryElements = indexElement.select("div[class=\"story\"]");
                Elements indexIntroductionElements = indexElement.select("div[class=\"introduction\"]");
                if (!indexChapterElements.isEmpty()) {
                    indexContentElements = indexChapterElements;
                } else if (!indexActElements.isEmpty()) {
                    indexContentElements = indexActElements;
                } else if (!indexFrontmatterElements.isEmpty()) {
                    indexContentElements = indexFrontmatterElements;
                } else if (!indexEssayElements.isEmpty()) {
                    indexContentElements = indexEssayElements;
                } else if (!contentsElements.isEmpty()) {
                    // http://books.framework_template.cc/b/berkeley/george/analyst.html
                    indexContentElements = contentsElements;
                } else if (!indexStoryElements.isEmpty()) {
                    // http://books.framework_template.cc/g/gogol/nikolai/g61v.html
                    indexContentElements = indexStoryElements;
                } else if (!bodyElements.isEmpty()) {
                    // http://books.framework_template.cc/g/gilman/charlotte_perkins/yellow.html
                    indexContentElements = bodyElements;
                } else if (!indexIntroductionElements.isEmpty()) {
                    // http://books.framework_template.cc/m/maupassant/guy/gosse/index.html
                    indexContentElements = indexIntroductionElements;
                }
                if (indexContentElements != null && !indexContentElements.isEmpty()) {// 内容放在首页的，表示是只有单章的，直接获取内容
                    StringBuilder content = new StringBuilder();
                    for (int i = 0; i < indexContentElements.size(); i++) {
                        JsoupUtil.updateImgSrcToAbsolute(indexContentElements.get(i));
                        content.append(indexContentElements.get(i).html());
                    }
                    if (!contentsElements.isEmpty() && !bodyElements.isEmpty()) {
                        for (int i = 0; i < bodyElements.size(); i++) {
                            JsoupUtil.updateImgSrcToAbsolute(bodyElements.get(i));
                            content.append(bodyElements.get(i).html());
                        }
                    }

                    // indexContentElement.select("div[class=\"header\"]").remove();
                    chaptersList.add(ChaptersVO.builder().title(book.getName()).content(removeHrefPageLink(content.toString())).build());
                } else if (!indexWorksElements.isEmpty()) {// 首页包含的是著作信息
                    if (recursive) {
                        Elements bookElements = indexWorksElements.select("a");
                        bookElements.forEach(e -> {
                            try {
                                handleOneBookElement(e, false);
                            } catch (Throwable t) {
                                log.error("Handle book exception happened:" + e.absUrl("href"), t);
                            }
                        });
                    }
                } else {// 处理多章节
                    String contentChapterUrl = null;
                    Element chapterElement = null;
                    Element navElement = indexElement.select("div[class=\"nav\"]").first();
                    if (navElement != null) {
                        Elements navLinkElements = navElement.select("a");
                        if (!navLinkElements.isEmpty()) {
                            for (int i = 0; i < navLinkElements.size(); i++) {
                                Element tableOfContentsElement = navLinkElements.get(i);
                                if (tableOfContentsElement.ownText().trim().equalsIgnoreCase("Table of Contents")) {
                                    contentChapterUrl = tableOfContentsElement.absUrl("href");
                                    if (contentChapterUrl.indexOf("framework_template.cc") < 0) {// 是其它链接
                                        contentChapterUrl = null;
                                    }else {
                                        break;
                                    }
                                }
                            }
                        }
                    }
                    if (contentChapterUrl != null) {
                        chapterElement = JsoupUtil.getPageElement(contentChapterUrl);
                    }
                    if (chapterElement == null) {
                        contentChapterUrl = url.replace(".html", "/contents.html");
                        chapterElement = JsoupUtil.getPageElement(contentChapterUrl);
                        if (chapterElement == null) {
                            contentChapterUrl = StringUtil.substringBeforeLast(url, "/") + "/contents.html";
                            chapterElement = JsoupUtil.getPageElement(contentChapterUrl);
                            if (chapterElement == null) {
                                log.warn("Can not get chapter page for chapter url:" + contentChapterUrl
                                         + ",original url:" + url);
                                return null;
                            }
                        }
                    }
                    Elements chapterElements = null;
                    Element chapterElements1 = chapterElement.select("div[class=\"contents\"]").first();
                    Element chapterElements2 = chapterElement.select("div[id=\"contents\"]").first();
                    Element chapterElements3 = chapterElement.select("div[class=\"frontmatter\"]").first();
                    if (chapterElements1 != null) {
                        chapterElements = chapterElements1.select("a");
                    }
                    if (chapterElements == null && chapterElements2 != null) {
                        chapterElements = chapterElements2.select("a");
                    }
                    if (chapterElements == null && chapterElements3 != null) {
                        chapterElements = chapterElements3.select("a");
                    }
                    if (chapterElements == null || chapterElements.size() <= 0) {
                        log.warn("Can not get chapters from chapter url:" + contentChapterUrl);
                        return null;
                    }
                    chapterElements.forEach(c -> {
                        ChaptersVO chapter = handleOneChapter(c);
                        if (chapter != null) {
                            chaptersList.add(chapter);
                        }
                    });
                }
                book.setChapters(chaptersList);
                return book;
            } catch (Exception e1) {
                log.error("Handle current book exception happened:" + url, e1);
            }
        }
        return null;
    }

    /**
     * 处理一章
     * 
     * @param chapterElement
     * @return
     */
    public ChaptersVO handleOneChapter(Element chapterElement) {

        String chapterTitle = chapterElement.ownText();
        String chapterUrl = chapterElement.attr("href");
        String chapterAbsUrl = chapterElement.absUrl("href");
        String sectionTag = null;
        if (chapterUrl.indexOf("#") > 0) {
            sectionTag = chapterUrl.substring(chapterUrl.indexOf("#") + 1);
        }
        try {

            if (chapterAbsUrl.endsWith(".pdf")) {// 是pdf文件
                return ChaptersVO.builder().title(chapterTitle).content("<a href='" + chapterAbsUrl + "'>"
                                                                        + chapterTitle + "</a>").build();
            }

            Element chapterContentElement = JsoupUtil.getPageElement(chapterAbsUrl);
            if (chapterContentElement == null) {
                return null;
            }
            Element chapterContentPartElement = null;
            Element prefaceElement = chapterContentElement.select("div[class=\"preface\"]").first();
            Element partElement = chapterContentElement.select("div[class=\"part\"]").first();
            Element contentChapterElement = chapterContentElement.select("div[class=\"chapter\"]").first();
            Element glossaryElement = chapterContentElement.select("div[class=\"glossary\"]").first();
            Element introductionElement = chapterContentElement.select("div[class=\"introduction\"]").first();
            Element bookElement = chapterContentElement.select("div[class=\"book\"]").first();
            Element appendixElement = chapterContentElement.select("div[class=\"appendix\"]").first();
            Element storyElement = chapterContentElement.select("div[class=\"story\"]").first();
            Element frontmatterElement = chapterContentElement.select("div[class=\"frontmatter\"]").first();
            Element actElement = chapterContentElement.select("div[class=\"act\"]").first();
            Element essayElement = chapterContentElement.select("div[class=\"essay\"]").first();
            Element cantoElement = chapterContentElement.select("div[class=\"canto\"]").first();
            if (prefaceElement != null) {
                chapterContentPartElement = prefaceElement;
            }else if (partElement != null) {
                chapterContentPartElement = partElement;
            }else if (glossaryElement != null) {
                chapterContentPartElement = glossaryElement;
            }else if (contentChapterElement != null) {
                chapterContentPartElement = contentChapterElement;
            }else if (introductionElement != null) {
                chapterContentPartElement = introductionElement;
            }else if (bookElement != null) {
                chapterContentPartElement = bookElement;
            }else if (appendixElement != null) {
                chapterContentPartElement = appendixElement;
            }else if (storyElement != null) {
                chapterContentPartElement = storyElement;
            }else if (frontmatterElement != null) {
                chapterContentPartElement = frontmatterElement;
            }else if (actElement != null) {
                chapterContentPartElement = actElement;
            }else if (essayElement != null) {
                chapterContentPartElement = essayElement;
            }else if (cantoElement != null) {
                chapterContentPartElement = cantoElement;
            }
            if (chapterContentPartElement == null) {
                log.warn("Get chapter content for url:" + chapterAbsUrl + " exception happened");
                return null;
            }
            if (!StringUtil.isEmpty(sectionTag)) {
                Element sectionEle = chapterContentPartElement.select("div[id=\"" + sectionTag + "\"]").first();
                if (sectionEle != null) {
                    chapterContentPartElement = sectionEle;
                }
            }
            JsoupUtil.updateImgSrcToAbsolute(chapterContentPartElement);
            return ChaptersVO.builder().title(chapterTitle).content(removeHrefPageLink(chapterContentPartElement.html())).build();
        } catch (Exception e) {
            log.error("handle one chapter exception, url:" + chapterAbsUrl, e);
        }
        return null;
    }

    /**
     * 保存Book到数据库
     * 
     * @param book
     */
    private void insert(BooksVO book) {
        final Books record = new Books();
        record.setAuthor(book.getAuthor());
        record.setChapters(book.getChapters().size());
        record.setIntroduce(book.getIntroduce());
        record.setKeywords(book.getKeywords());
        record.setLanguage(book.getLanguage());
        String name = book.getName();
        if (name.length() >= 256) {
            name = name.substring(0, 128);
        }
        record.setName(name);
        record.setPublished(book.getPublished());
        record.setCover(book.getCover());
        log.info("insert book");
        bookService.insert(record);
        for (int index = 0; index < book.getChapters().size(); index++) {
            ChaptersVO c = book.getChapters().get(index);
            BooksChapters chapter = new BooksChapters();
            chapter.setBookId(record.getId());
            name = c.getTitle();
            if (name.length() >= 256) {
                name = name.substring(0, 128);
            }
            chapter.setName(name);
            chapter.setContent(c.getContent());
            int chapterIndex = index + 1;
            chapter.setChapterIndex(chapterIndex);
            log.info("insert book chapter");
            bookChapterService.insert(chapter);
        }
    }

    /**
     * 將超链接中代表当前页面的地址给删除，只保留相应的Tag，后续使用自定义页面时才可扩展。如：<br>
     * 原链接地址为preface.html#xxxx，只保留#xxxx
     * 
     * @param str
     * @return
     */
    private String removeHrefPageLink(String str) {
        if (StringUtil.isEmpty(str)) {
            return str;
        }
        str = str.replaceAll("href=\"[0-9A-Za-z:/[-]_#[?][=][.][&]]*.html#", "href=\"#");
        return str;
    }
}
